#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2024/2/4 17:56
# @Author  : 王凯
# @File    : gen_scrapy_spider.py
# @Project : scrapy_spider
from pathlib import Path

import yaml
from jinja2 import Template

from apps.tax_policy.tax_policy.std_template.config_spider_entry import ConfigSpiderData, Constants
from utils import tools


class ScrapyGenerator:
    def __init__(self, config_data):
        self.config_data = config_data
        self.static_self_item_key = ["source", "province", "city", "county", "park"]
        with open(f"{Path(__file__).parent}/spider.templ", "r", encoding="utf-8") as f:
            self.render_templ = f.read()

    def set_file_variable(self, key, value):
        if key in self.render_templ:
            self.render_templ = Template(self.render_templ).render(**{key: value})

    def generate(self):
        self.process_spider()
        return self.render_templ + "\n"

    def process_spider(self):
        str_parser = ""
        for stage in self.config_data.stages:
            stage_name = stage.name
            stage_str = self.get_parser_string(stage_name, stage)
            str_parser += stage_str

        render_dict = {
            Constants.AnchorStartStage: f"parse_{self.get_start_stage_name(self.config_data)}",
            Constants.AnchorStartUrl: self.config_data.start_url,
            Constants.AnchorParsers: str_parser,
            Constants.AnchorName: self.config_data.name,
            Constants.NAME_NO_LINE: tools.key2hump(self.config_data.name),
        }

        if self.get_settings(self.config_data):
            render_dict.update(
                {
                    Constants.CUSTOM_SETTINGS: self.pad_code(
                        f"custom_settings = {self.get_settings(self.config_data)}", 0
                    ),
                }
            )

        if self.get_static_key(self.config_data):
            static_key_str = self.pad_code("", num=0)
            for k, v in self.get_static_key(self.config_data).items():
                static_key_str += self.pad_code(f"{k} = '{v}'", 1)
            if static_key_str:
                render_dict.update({Constants.STATIC_SELF_ITEM: static_key_str.rstrip()})

        self.render_templ = Template(self.render_templ).render(**render_dict)

        return None

    def get_parser_string(self, stage_name, stage):
        str_def = self.pad_code(f"def parse_{stage_name}(self, response, **kwargs):", 1)

        if stage.is_list:
            str_parse = self.get_list_parser_string(stage_name, stage)
        else:
            str_parse = self.get_non_list_parser_string(stage_name, stage)

        return f"{str_def}{str_parse}"

    def pad_code(self, string, num):
        res = ""
        for i in range(num):
            res += "    "
        res += string
        res += "\n"
        return res

    def get_non_list_parser_string(self, stage_name, stage):
        str_result = ""
        str_result += self.pad_code(
            "item = Item() if response.meta.get('item') is None else response.meta.get('item')", 2
        )

        for f in stage.fields:
            line = f"item['{f.name}'] = response.{self.get_extract_string_from_field(f)}"
            line = self.pad_code(line, 2)
            str_result += line

        if self.get_next_stage_field(stage) is not None:
            f = self.get_next_stage_field(stage)
            str_result += self.pad_code(
                f"yield response.follow(item['{f.name}'], callback=self.parse_{f.next_stage}, meta={'item': item})",
                2,
            )
        else:
            for key in self.config_data.static_self_item_key:
                if key == 'url':
                    continue
                str_result += self.pad_code(f"item['{key}'] = self.{key}", 2)
            str_result += self.pad_code("yield item", 2)

        str_result += self.pad_code("", 0)
        return str_result

    def get_list_parser_string(self, stage_name, stage):
        str_result = ""
        str_result += self.pad_code("prev_item = response.meta.get('item')", 2)
        str_result += self.pad_code(f"for elem in response.{self.get_list_string(stage)}:", 2)
        str_result += self.pad_code("item = Item()", 3)

        for f in stage.fields:
            line = rf"item['{f.name}'] = elem.{self.get_extract_string_from_field(f)}"
            line = self.pad_code(line, 3)
            str_result += line
            if "url" in f.name:
                url_line = f"item['{f.name}'] = response.urljoin(item['{f.name}'])"
                url_line = self.pad_code(url_line, 3)
                str_result += url_line

        str_result += self.pad_code("if prev_item is not None:", 3)
        str_result += self.pad_code("for key, value in prev_item.items():", 4)
        str_result += self.pad_code("item[key] = value", 5)

        if self.get_next_stage_field(stage) is not None:
            f = self.get_next_stage_field(stage)
            str_result += self.pad_code(f"if not item['{f.name}']:", 3)
            str_result += self.pad_code("continue", 4)
            str_result += self.pad_code(
                f"yield response.follow(item['{f.name}'], callback=self.parse_{f.next_stage}, " "meta={'item': item})",
                3,
            )
        else:
            str_result += self.pad_code("yield item", 3)

        if (stage.page_css != "" and stage.page_css is not None) or (
            stage.page_xpath != "" and stage.page_xpath is not None
        ):
            str_result += self.pad_code(f"next_url = response.{self.get_extract_string_from_stage(stage)}", 2)
            str_result += self.pad_code(
                f"yield response.follow(next_url, callback=self.parse_{stage_name}," " meta={'item': prev_item})",
                2,
            )
        elif (stage.page_loop_css != "" and stage.page_loop_css is not None) or (stage.page_loop_xpath != "" and
                                                                                 stage.page_loop_xpath is not None):
            page_loop_page_size = 1
            if stage.page_loop_page_size != "" and stage.page_loop_page_size is not None:
                page_loop_page_size = int(stage.page_loop_page_size)
            str_result += self.pad_code('if response.meta.get("is_next") is not False:', 2)
            str_result += self.pad_code(f"total_page = response.{self.get_extract_string_loop_from_stage(stage)}", 3)
            str_result += self.pad_code(f"for page_num in range(1, int(total_page) // {page_loop_page_size}):", 3)
            str_result += self.pad_code(
                f'yield response.follow(re.sub(r"{stage.page_loop_sub_src}", '
                f'r"{stage.page_loop_sub_target}".format(page_num), response.url), '
                f"callback=self.parse_{stage_name}, "
                "meta={'item': prev_item, 'is_next': False})",
                4,
            )

        str_result += self.pad_code("", 0)
        return str_result

    def get_all_fields(self):
        return self.config_data.dict().keys()

    def get_start_stage_name(self, config_data):
        return config_data.start_stage

    def get_settings(self, config_data):
        return config_data.settings

    def get_static_key(self, config_data):
        item = {}
        for key in config_data.static_self_item_key:
            item.update({key: getattr(config_data, key) or ""})
        return item

    def get_next_stage_field(self, stage):
        for f in stage.fields:
            if f.next_stage != "" and f.next_stage is not None:
                return f
        return None

    def get_extract_string_from_field(self, f):
        if f.css != "" and f.css is not None:
            if f.attr == "":
                return (
                    rf'css("""{f.css}::text""").re_first(r"""{f.re_first}""")'
                    if f.re_first != ""
                    else f'css("""{f.css}::text""").get()'
                )
            else:
                return (
                    rf'css("""{f.css}::attr("{f.attr}")""").re_first(r"""{f.re_first}""")'
                    if f.re_first != ""
                    else f'css("""{f.css}::attr("{f.attr}")""").get()'
                )
        else:
            if f.attr == "":
                return (
                    rf'xpath("""{f.xpath}""").re_first(r"""{f.re_first}""")'
                    if f.re_first != ""
                    else f'xpath("""{f.xpath}""").get()'
                )
            else:
                return (
                    rf'xpath("""{f.xpath}/@{f.attr}""").re_first(r"""{f.re_first}""")'
                    if f.re_first != ""
                    else f'xpath("""{f.xpath}/@{f.attr}""").get()'
                )

    def get_extract_string_from_stage(self, stage):
        page_attr = "href"
        if stage.page_attr != "" and stage.page_attr is not None:
            page_attr = stage.page_attr

        if stage.page_css != "" and stage.page_css is not None:
            return f'css("""{stage.page_css}::attr("{page_attr}")""")'
        else:
            return f'xpath("""{stage.page_xpath}/@{page_attr}""")'

    def get_extract_string_loop_from_stage(self, stage):
        css_template = 'css("""{}{}{}""")'
        xpath_template = 'xpath("""{}{}{}""")'

        page_loop_attr = f"::{stage.page_loop_attr}" if stage.page_loop_attr else ""

        if stage.page_loop_css:
            return self._build_css_selector(css_template, stage.page_loop_css, page_loop_attr, stage.page_loop_re_first)
        else:
            return self._build_xpath_selector(
                xpath_template, stage.page_loop_xpath, page_loop_attr, stage.page_loop_re_first
            )

    def _build_css_selector(self, template, css, attr, re_first):
        selector = template.format(css, attr, "").rstrip(":")
        return self._add_re_first(selector, re_first)

    def _build_xpath_selector(self, template, xpath, attr, re_first):
        selector = template.format(xpath, attr, "/@" if attr else "").rstrip("/@")
        return self._add_re_first(selector, re_first)

    def _add_re_first(self, selector, re_first):
        return f'{selector}.re_first(r"""{re_first}""")' if re_first else f"{selector}.get()"

    def get_list_string(self, stage):
        if stage.list_css != "" and stage.list_css is not None:
            return f'css("{stage.list_css}")'
        else:
            return f'xpath("{stage.list_xpath}")'


if __name__ == "__main__":
    with open("example.yaml", "r", encoding="utf-8") as file:
        data = yaml.safe_load(file)
        data = ConfigSpiderData(**data)
    print(data.dict())
    spider_file_content = ScrapyGenerator(data).generate()
    with open("example.py", "w", encoding="utf-8") as fs:
        fs.write(spider_file_content)
